{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "from fastai.vision import *\n",
    "import torch\n",
    "import warnings\n",
    "from torchsummary import summary\n",
    "# from models.custom_resnet import _resnet, Bottleneck\n",
    "from models.resnet_cifar import *\n",
    "from utils import _get_accuracy\n",
    "torch.cuda.set_device(0)\n",
    "torch.manual_seed(1)\n",
    "torch.cuda.manual_seed(1)\n",
    "\n",
    "# stage should be in 0 to 5 (5 for classifier stage)\n",
    "hyper_params = {\n",
    "    \"stage\": 1,\n",
    "    \"repeated\": 0,\n",
    "    \"num_classes\": 10,\n",
    "    \"batch_size\": 64,\n",
    "    \"num_epochs\": 2,\n",
    "    \"learning_rate\": 1e-1\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = untar_data(URLs.CIFAR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "tfms = get_transforms(do_flip=False)\n",
    "data = ImageDataBunch.from_folder(path, train = 'train', valid = 'test', bs = hyper_params[\"batch_size\"], size = 32, ds_tfms = tfms).normalize(cifar_stats)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model on GPU\n",
      "conv1.weight torch.Size([16, 3, 3, 3])\n",
      "False\n",
      "bn1.weight torch.Size([16])\n",
      "False\n",
      "bn1.bias torch.Size([16])\n",
      "False\n",
      "layer1.0.conv1.weight torch.Size([16, 16, 3, 3])\n",
      "True\n",
      "layer1.0.bn1.weight torch.Size([16])\n",
      "True\n",
      "layer1.0.bn1.bias torch.Size([16])\n",
      "True\n",
      "layer1.0.conv2.weight torch.Size([16, 16, 3, 3])\n",
      "True\n",
      "layer1.0.bn2.weight torch.Size([16])\n",
      "True\n",
      "layer1.0.bn2.bias torch.Size([16])\n",
      "True\n",
      "layer1.1.conv1.weight torch.Size([16, 16, 3, 3])\n",
      "True\n",
      "layer1.1.bn1.weight torch.Size([16])\n",
      "True\n",
      "layer1.1.bn1.bias torch.Size([16])\n",
      "True\n",
      "layer1.1.conv2.weight torch.Size([16, 16, 3, 3])\n",
      "True\n",
      "layer1.1.bn2.weight torch.Size([16])\n",
      "True\n",
      "layer1.1.bn2.bias torch.Size([16])\n",
      "True\n",
      "layer2.0.conv1.weight torch.Size([32, 16, 3, 3])\n",
      "False\n",
      "layer2.0.bn1.weight torch.Size([32])\n",
      "False\n",
      "layer2.0.bn1.bias torch.Size([32])\n",
      "False\n",
      "layer2.0.conv2.weight torch.Size([32, 32, 3, 3])\n",
      "False\n",
      "layer2.0.bn2.weight torch.Size([32])\n",
      "False\n",
      "layer2.0.bn2.bias torch.Size([32])\n",
      "False\n",
      "layer2.0.downsample.0.weight torch.Size([32, 16, 1, 1])\n",
      "False\n",
      "layer2.0.downsample.1.weight torch.Size([32])\n",
      "False\n",
      "layer2.0.downsample.1.bias torch.Size([32])\n",
      "False\n",
      "layer2.1.conv1.weight torch.Size([32, 32, 3, 3])\n",
      "False\n",
      "layer2.1.bn1.weight torch.Size([32])\n",
      "False\n",
      "layer2.1.bn1.bias torch.Size([32])\n",
      "False\n",
      "layer2.1.conv2.weight torch.Size([32, 32, 3, 3])\n",
      "False\n",
      "layer2.1.bn2.weight torch.Size([32])\n",
      "False\n",
      "layer2.1.bn2.bias torch.Size([32])\n",
      "False\n",
      "layer3.0.conv1.weight torch.Size([64, 32, 3, 3])\n",
      "False\n",
      "layer3.0.bn1.weight torch.Size([64])\n",
      "False\n",
      "layer3.0.bn1.bias torch.Size([64])\n",
      "False\n",
      "layer3.0.conv2.weight torch.Size([64, 64, 3, 3])\n",
      "False\n",
      "layer3.0.bn2.weight torch.Size([64])\n",
      "False\n",
      "layer3.0.bn2.bias torch.Size([64])\n",
      "False\n",
      "layer3.0.downsample.0.weight torch.Size([64, 32, 1, 1])\n",
      "False\n",
      "layer3.0.downsample.1.weight torch.Size([64])\n",
      "False\n",
      "layer3.0.downsample.1.bias torch.Size([64])\n",
      "False\n",
      "layer3.1.conv1.weight torch.Size([64, 64, 3, 3])\n",
      "False\n",
      "layer3.1.bn1.weight torch.Size([64])\n",
      "False\n",
      "layer3.1.bn1.bias torch.Size([64])\n",
      "False\n",
      "layer3.1.conv2.weight torch.Size([64, 64, 3, 3])\n",
      "False\n",
      "layer3.1.bn2.weight torch.Size([64])\n",
      "False\n",
      "layer3.1.bn2.bias torch.Size([64])\n",
      "False\n",
      "fc.weight torch.Size([10, 64])\n",
      "False\n",
      "fc.bias torch.Size([10])\n",
      "False\n"
     ]
    }
   ],
   "source": [
    "model = resnet26_cifar()\n",
    "model.cpu()\n",
    "model.load_state_dict(torch.load('../saved_models/cifar10/resnet26_pretraining/model0.pt', map_location = 'cpu'))\n",
    "model = model.cuda()\n",
    "# Freezing weights of teacher\n",
    "for name, param in model.named_parameters() : \n",
    "    param.requires_grad = False\n",
    "\n",
    "net = resnet14_cifar()\n",
    "\n",
    "if torch.cuda.is_available() : \n",
    "    net = net.cuda()\n",
    "    model = model.cuda()\n",
    "    print('Model on GPU')\n",
    "    \n",
    "for name, param in net.named_parameters() : \n",
    "    print(name, param.shape)\n",
    "    param.requires_grad = False\n",
    "    if name[5] == str(hyper_params['stage']) and hyper_params['stage'] != 0 :\n",
    "        param.requires_grad = True\n",
    "    elif (name[0] == 'b' or name[0] == 'c') and hyper_params['stage'] == 0 : \n",
    "        param.requires_grad = True\n",
    "    elif name[0] == 'f' and hyper_params['stage'] == 4 : \n",
    "        param.requires_grad = True\n",
    "    print(param.requires_grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ResNet_Cifar(\n",
       "  (conv1): Conv2d(3, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "  (bn1): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (relu2): ReLU(inplace)\n",
       "  (layer1): Sequential(\n",
       "    (0): BasicBlock(\n",
       "      (conv1): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace)\n",
       "      (conv2): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (1): BasicBlock(\n",
       "      (conv1): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace)\n",
       "      (conv2): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "  )\n",
       "  (layer2): Sequential(\n",
       "    (0): BasicBlock(\n",
       "      (conv1): Conv2d(16, 32, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace)\n",
       "      (conv2): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (downsample): Sequential(\n",
       "        (0): Conv2d(16, 32, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
       "        (1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "    )\n",
       "    (1): BasicBlock(\n",
       "      (conv1): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace)\n",
       "      (conv2): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "  )\n",
       "  (layer3): Sequential(\n",
       "    (0): BasicBlock(\n",
       "      (conv1): Conv2d(32, 64, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace)\n",
       "      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (downsample): Sequential(\n",
       "        (0): Conv2d(32, 64, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
       "        (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "    )\n",
       "    (1): BasicBlock(\n",
       "      (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace)\n",
       "      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "  )\n",
       "  (avgpool): AvgPool2d(kernel_size=8, stride=1, padding=0)\n",
       "  (fc): Linear(in_features=64, out_features=10, bias=True)\n",
       "  (relu): ReLU(inplace)\n",
       ")"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------------------------\n",
      "        Layer (type)               Output Shape         Param #\n",
      "================================================================\n",
      "            Conv2d-1           [-1, 16, 32, 32]             432\n",
      "       BatchNorm2d-2           [-1, 16, 32, 32]              32\n",
      "              ReLU-3           [-1, 16, 32, 32]               0\n",
      "            Conv2d-4           [-1, 16, 32, 32]           2,304\n",
      "       BatchNorm2d-5           [-1, 16, 32, 32]              32\n",
      "              ReLU-6           [-1, 16, 32, 32]               0\n",
      "            Conv2d-7           [-1, 16, 32, 32]           2,304\n",
      "       BatchNorm2d-8           [-1, 16, 32, 32]              32\n",
      "              ReLU-9           [-1, 16, 32, 32]               0\n",
      "       BasicBlock-10           [-1, 16, 32, 32]               0\n",
      "           Conv2d-11           [-1, 16, 32, 32]           2,304\n",
      "      BatchNorm2d-12           [-1, 16, 32, 32]              32\n",
      "             ReLU-13           [-1, 16, 32, 32]               0\n",
      "           Conv2d-14           [-1, 16, 32, 32]           2,304\n",
      "      BatchNorm2d-15           [-1, 16, 32, 32]              32\n",
      "             ReLU-16           [-1, 16, 32, 32]               0\n",
      "       BasicBlock-17           [-1, 16, 32, 32]               0\n",
      "           Conv2d-18           [-1, 32, 16, 16]           4,608\n",
      "      BatchNorm2d-19           [-1, 32, 16, 16]              64\n",
      "             ReLU-20           [-1, 32, 16, 16]               0\n",
      "           Conv2d-21           [-1, 32, 16, 16]           9,216\n",
      "      BatchNorm2d-22           [-1, 32, 16, 16]              64\n",
      "           Conv2d-23           [-1, 32, 16, 16]             512\n",
      "      BatchNorm2d-24           [-1, 32, 16, 16]              64\n",
      "             ReLU-25           [-1, 32, 16, 16]               0\n",
      "       BasicBlock-26           [-1, 32, 16, 16]               0\n",
      "           Conv2d-27           [-1, 32, 16, 16]           9,216\n",
      "      BatchNorm2d-28           [-1, 32, 16, 16]              64\n",
      "             ReLU-29           [-1, 32, 16, 16]               0\n",
      "           Conv2d-30           [-1, 32, 16, 16]           9,216\n",
      "      BatchNorm2d-31           [-1, 32, 16, 16]              64\n",
      "             ReLU-32           [-1, 32, 16, 16]               0\n",
      "       BasicBlock-33           [-1, 32, 16, 16]               0\n",
      "           Conv2d-34             [-1, 64, 8, 8]          18,432\n",
      "      BatchNorm2d-35             [-1, 64, 8, 8]             128\n",
      "             ReLU-36             [-1, 64, 8, 8]               0\n",
      "           Conv2d-37             [-1, 64, 8, 8]          36,864\n",
      "      BatchNorm2d-38             [-1, 64, 8, 8]             128\n",
      "           Conv2d-39             [-1, 64, 8, 8]           2,048\n",
      "      BatchNorm2d-40             [-1, 64, 8, 8]             128\n",
      "             ReLU-41             [-1, 64, 8, 8]               0\n",
      "       BasicBlock-42             [-1, 64, 8, 8]               0\n",
      "           Conv2d-43             [-1, 64, 8, 8]          36,864\n",
      "      BatchNorm2d-44             [-1, 64, 8, 8]             128\n",
      "             ReLU-45             [-1, 64, 8, 8]               0\n",
      "           Conv2d-46             [-1, 64, 8, 8]          36,864\n",
      "      BatchNorm2d-47             [-1, 64, 8, 8]             128\n",
      "             ReLU-48             [-1, 64, 8, 8]               0\n",
      "       BasicBlock-49             [-1, 64, 8, 8]               0\n",
      "        AvgPool2d-50             [-1, 64, 1, 1]               0\n",
      "           Linear-51                   [-1, 10]             650\n",
      "================================================================\n",
      "Total params: 175,258\n",
      "Trainable params: 175,258\n",
      "Non-trainable params: 0\n",
      "----------------------------------------------------------------\n",
      "Input size (MB): 0.01\n",
      "Forward/backward pass size (MB): 3.63\n",
      "Params size (MB): 0.67\n",
      "Estimated Total Size (MB): 4.31\n",
      "----------------------------------------------------------------\n",
      "----------------------------------------------------------------\n",
      "        Layer (type)               Output Shape         Param #\n",
      "================================================================\n",
      "            Conv2d-1           [-1, 16, 32, 32]             432\n",
      "       BatchNorm2d-2           [-1, 16, 32, 32]              32\n",
      "              ReLU-3           [-1, 16, 32, 32]               0\n",
      "            Conv2d-4           [-1, 16, 32, 32]           2,304\n",
      "       BatchNorm2d-5           [-1, 16, 32, 32]              32\n",
      "              ReLU-6           [-1, 16, 32, 32]               0\n",
      "            Conv2d-7           [-1, 16, 32, 32]           2,304\n",
      "       BatchNorm2d-8           [-1, 16, 32, 32]              32\n",
      "              ReLU-9           [-1, 16, 32, 32]               0\n",
      "       BasicBlock-10           [-1, 16, 32, 32]               0\n",
      "           Conv2d-11           [-1, 16, 32, 32]           2,304\n",
      "      BatchNorm2d-12           [-1, 16, 32, 32]              32\n",
      "             ReLU-13           [-1, 16, 32, 32]               0\n",
      "           Conv2d-14           [-1, 16, 32, 32]           2,304\n",
      "      BatchNorm2d-15           [-1, 16, 32, 32]              32\n",
      "             ReLU-16           [-1, 16, 32, 32]               0\n",
      "       BasicBlock-17           [-1, 16, 32, 32]               0\n",
      "           Conv2d-18           [-1, 16, 32, 32]           2,304\n",
      "      BatchNorm2d-19           [-1, 16, 32, 32]              32\n",
      "             ReLU-20           [-1, 16, 32, 32]               0\n",
      "           Conv2d-21           [-1, 16, 32, 32]           2,304\n",
      "      BatchNorm2d-22           [-1, 16, 32, 32]              32\n",
      "             ReLU-23           [-1, 16, 32, 32]               0\n",
      "       BasicBlock-24           [-1, 16, 32, 32]               0\n",
      "           Conv2d-25           [-1, 16, 32, 32]           2,304\n",
      "      BatchNorm2d-26           [-1, 16, 32, 32]              32\n",
      "             ReLU-27           [-1, 16, 32, 32]               0\n",
      "           Conv2d-28           [-1, 16, 32, 32]           2,304\n",
      "      BatchNorm2d-29           [-1, 16, 32, 32]              32\n",
      "             ReLU-30           [-1, 16, 32, 32]               0\n",
      "       BasicBlock-31           [-1, 16, 32, 32]               0\n",
      "           Conv2d-32           [-1, 32, 16, 16]           4,608\n",
      "      BatchNorm2d-33           [-1, 32, 16, 16]              64\n",
      "             ReLU-34           [-1, 32, 16, 16]               0\n",
      "           Conv2d-35           [-1, 32, 16, 16]           9,216\n",
      "      BatchNorm2d-36           [-1, 32, 16, 16]              64\n",
      "           Conv2d-37           [-1, 32, 16, 16]             512\n",
      "      BatchNorm2d-38           [-1, 32, 16, 16]              64\n",
      "             ReLU-39           [-1, 32, 16, 16]               0\n",
      "       BasicBlock-40           [-1, 32, 16, 16]               0\n",
      "           Conv2d-41           [-1, 32, 16, 16]           9,216\n",
      "      BatchNorm2d-42           [-1, 32, 16, 16]              64\n",
      "             ReLU-43           [-1, 32, 16, 16]               0\n",
      "           Conv2d-44           [-1, 32, 16, 16]           9,216\n",
      "      BatchNorm2d-45           [-1, 32, 16, 16]              64\n",
      "             ReLU-46           [-1, 32, 16, 16]               0\n",
      "       BasicBlock-47           [-1, 32, 16, 16]               0\n",
      "           Conv2d-48           [-1, 32, 16, 16]           9,216\n",
      "      BatchNorm2d-49           [-1, 32, 16, 16]              64\n",
      "             ReLU-50           [-1, 32, 16, 16]               0\n",
      "           Conv2d-51           [-1, 32, 16, 16]           9,216\n",
      "      BatchNorm2d-52           [-1, 32, 16, 16]              64\n",
      "             ReLU-53           [-1, 32, 16, 16]               0\n",
      "       BasicBlock-54           [-1, 32, 16, 16]               0\n",
      "           Conv2d-55           [-1, 32, 16, 16]           9,216\n",
      "      BatchNorm2d-56           [-1, 32, 16, 16]              64\n",
      "             ReLU-57           [-1, 32, 16, 16]               0\n",
      "           Conv2d-58           [-1, 32, 16, 16]           9,216\n",
      "      BatchNorm2d-59           [-1, 32, 16, 16]              64\n",
      "             ReLU-60           [-1, 32, 16, 16]               0\n",
      "       BasicBlock-61           [-1, 32, 16, 16]               0\n",
      "           Conv2d-62             [-1, 64, 8, 8]          18,432\n",
      "      BatchNorm2d-63             [-1, 64, 8, 8]             128\n",
      "             ReLU-64             [-1, 64, 8, 8]               0\n",
      "           Conv2d-65             [-1, 64, 8, 8]          36,864\n",
      "      BatchNorm2d-66             [-1, 64, 8, 8]             128\n",
      "           Conv2d-67             [-1, 64, 8, 8]           2,048\n",
      "      BatchNorm2d-68             [-1, 64, 8, 8]             128\n",
      "             ReLU-69             [-1, 64, 8, 8]               0\n",
      "       BasicBlock-70             [-1, 64, 8, 8]               0\n",
      "           Conv2d-71             [-1, 64, 8, 8]          36,864\n",
      "      BatchNorm2d-72             [-1, 64, 8, 8]             128\n",
      "             ReLU-73             [-1, 64, 8, 8]               0\n",
      "           Conv2d-74             [-1, 64, 8, 8]          36,864\n",
      "      BatchNorm2d-75             [-1, 64, 8, 8]             128\n",
      "             ReLU-76             [-1, 64, 8, 8]               0\n",
      "       BasicBlock-77             [-1, 64, 8, 8]               0\n",
      "           Conv2d-78             [-1, 64, 8, 8]          36,864\n",
      "      BatchNorm2d-79             [-1, 64, 8, 8]             128\n",
      "             ReLU-80             [-1, 64, 8, 8]               0\n",
      "           Conv2d-81             [-1, 64, 8, 8]          36,864\n",
      "      BatchNorm2d-82             [-1, 64, 8, 8]             128\n",
      "             ReLU-83             [-1, 64, 8, 8]               0\n",
      "       BasicBlock-84             [-1, 64, 8, 8]               0\n",
      "           Conv2d-85             [-1, 64, 8, 8]          36,864\n",
      "      BatchNorm2d-86             [-1, 64, 8, 8]             128\n",
      "             ReLU-87             [-1, 64, 8, 8]               0\n",
      "           Conv2d-88             [-1, 64, 8, 8]          36,864\n",
      "      BatchNorm2d-89             [-1, 64, 8, 8]             128\n",
      "             ReLU-90             [-1, 64, 8, 8]               0\n",
      "       BasicBlock-91             [-1, 64, 8, 8]               0\n",
      "        AvgPool2d-92             [-1, 64, 1, 1]               0\n",
      "           Linear-93                   [-1, 10]             650\n",
      "================================================================\n",
      "Total params: 369,690\n",
      "Trainable params: 0\n",
      "Non-trainable params: 369,690\n",
      "----------------------------------------------------------------\n",
      "Input size (MB): 0.01\n",
      "Forward/backward pass size (MB): 6.69\n",
      "Params size (MB): 1.41\n",
      "Estimated Total Size (MB): 8.11\n",
      "----------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "# x, y = next(iter(data.train_dl))\n",
    "# net(torch.autograd.Variable(x).cuda())\n",
    "summary(net, (3, 32, 32))\n",
    "summary(model, (3, 32, 32))\n",
    "# print(learn.summary())\n",
    "# net.layer1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SaveFeatures :\n",
    "    def __init__(self, m) : \n",
    "        self.handle = m.register_forward_hook(self.hook_fn)\n",
    "    def hook_fn(self, m, inp, outp) : \n",
    "        self.features = outp\n",
    "    def remove(self) :\n",
    "        self.handle.remove()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "sf = [SaveFeatures(m) for m in [model.relu2, model.layer1, model.layer2, model.layer3]]\n",
    "sf2 = [SaveFeatures(m) for m in [net.relu2, net.layer1, net.layer2, net.layer3]]\n",
    "\n",
    "## For testing hooks \n",
    "x, y = next(iter(data.train_dl))\n",
    "x = torch.autograd.Variable(x).cuda()\n",
    "out1 = model(x)\n",
    "out2 = net(x)\n",
    "for i in range(4) : \n",
    "#     print('resnet50 : ', sf[i].features.shape)\n",
    "#     print('custom model : ', sf2[i].features.shape)\n",
    "    assert(sf[i].features.shape == sf2[i].features.shape)\n",
    "del x, y\n",
    "del out1, out2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Stage-wise training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch =  1  step =  50  of total steps  805  loss =  0.15482433140277863\n",
      "epoch =  1  step =  100  of total steps  805  loss =  0.12196048349142075\n",
      "epoch =  1  step =  150  of total steps  805  loss =  0.14741818606853485\n",
      "epoch =  1  step =  200  of total steps  805  loss =  0.156645268201828\n",
      "epoch =  1  step =  250  of total steps  805  loss =  0.1329699158668518\n",
      "epoch =  1  step =  300  of total steps  805  loss =  0.1337122768163681\n",
      "epoch =  1  step =  350  of total steps  805  loss =  0.11640556156635284\n",
      "epoch =  1  step =  400  of total steps  805  loss =  0.11518991738557816\n",
      "epoch =  1  step =  450  of total steps  805  loss =  0.11609910428524017\n",
      "epoch =  1  step =  500  of total steps  805  loss =  0.11605231463909149\n",
      "epoch =  1  step =  550  of total steps  805  loss =  0.10304179042577744\n",
      "epoch =  1  step =  600  of total steps  805  loss =  0.10243804007768631\n",
      "epoch =  1  step =  650  of total steps  805  loss =  0.10527341067790985\n",
      "epoch =  1  step =  700  of total steps  805  loss =  0.10937318950891495\n",
      "epoch =  1  step =  750  of total steps  805  loss =  0.08963960409164429\n",
      "epoch =  1  step =  800  of total steps  805  loss =  0.09278739243745804\n",
      "epoch :  1  /  2  | TL :  0.12025123917908402  | VL :  0.08692677388899028\n",
      "saving model\n",
      "epoch =  2  step =  50  of total steps  805  loss =  0.08528214693069458\n",
      "epoch =  2  step =  100  of total steps  805  loss =  0.08195324242115021\n",
      "epoch =  2  step =  150  of total steps  805  loss =  0.07963912934064865\n",
      "epoch =  2  step =  200  of total steps  805  loss =  0.08056091517210007\n",
      "epoch =  2  step =  250  of total steps  805  loss =  0.08498793095350266\n",
      "epoch =  2  step =  300  of total steps  805  loss =  0.08090215921401978\n",
      "epoch =  2  step =  350  of total steps  805  loss =  0.07447414845228195\n",
      "epoch =  2  step =  400  of total steps  805  loss =  0.06504533439874649\n",
      "epoch =  2  step =  450  of total steps  805  loss =  0.06957031041383743\n",
      "epoch =  2  step =  500  of total steps  805  loss =  0.06390005350112915\n",
      "epoch =  2  step =  550  of total steps  805  loss =  0.06317362189292908\n",
      "epoch =  2  step =  600  of total steps  805  loss =  0.06319869309663773\n",
      "epoch =  2  step =  650  of total steps  805  loss =  0.06257843226194382\n",
      "epoch =  2  step =  700  of total steps  805  loss =  0.05500667914748192\n",
      "epoch =  2  step =  750  of total steps  805  loss =  0.06255189329385757\n",
      "epoch =  2  step =  800  of total steps  805  loss =  0.05196793004870415\n",
      "epoch :  2  /  2  | TL :  0.07195023581940935  | VL :  0.055773036438040435\n",
      "saving model\n"
     ]
    }
   ],
   "source": [
    "if hyper_params['stage'] == 0 : \n",
    "    filename = '../saved_models/large_stage' + str(hyper_params['stage']) + '/model' + str(hyper_params['repeated']) + '.pt'\n",
    "else : \n",
    "    filename = '../saved_models/large_stage' + str(hyper_params['stage'] + 1) + '/model' + str(hyper_params['repeated']) + '.pt'\n",
    "optimizer = torch.optim.Adam(net.parameters(), lr = hyper_params[\"learning_rate\"])\n",
    "total_step = len(data.train_ds) // hyper_params[\"batch_size\"]\n",
    "train_loss_list = list()\n",
    "val_loss_list = list()\n",
    "min_val = 100\n",
    "for epoch in range(hyper_params[\"num_epochs\"]):\n",
    "    trn = []\n",
    "    net.train()\n",
    "    for i, (images, labels) in enumerate(data.train_dl) :\n",
    "        if torch.cuda.is_available():\n",
    "            images = torch.autograd.Variable(images).cuda().float()\n",
    "            labels = torch.autograd.Variable(labels).cuda()\n",
    "        else : \n",
    "            images = torch.autograd.Variable(images).float()\n",
    "            labels = torch.autograd.Variable(labels)\n",
    "\n",
    "        y_pred = net(images)\n",
    "        y_pred2 = mdl(images)\n",
    "\n",
    "        loss = F.mse_loss(sf2[hyper_params[\"stage\"]].features, sf[hyper_params[\"stage\"]].features)\n",
    "        trn.append(loss.item())\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "#         torch.nn.utils.clip_grad_value_(net.parameters(), 10)\n",
    "        optimizer.step()\n",
    "\n",
    "        if i % 50 == 49 :\n",
    "            print('epoch = ', epoch + 1, ' step = ', i + 1, ' of total steps ', total_step, ' loss = ', loss.item())\n",
    "\n",
    "    train_loss = (sum(trn) / len(trn))\n",
    "    train_loss_list.append(train_loss)\n",
    "\n",
    "    net.eval()\n",
    "    val = []\n",
    "    with torch.no_grad() :\n",
    "        for i, (images, labels) in enumerate(data.valid_dl) :\n",
    "            if torch.cuda.is_available():\n",
    "                images = torch.autograd.Variable(images).cuda().float()\n",
    "                labels = torch.autograd.Variable(labels).cuda()\n",
    "            else : \n",
    "                images = torch.autograd.Variable(images).float()\n",
    "                labels = torch.autograd.Variable(labels)\n",
    "\n",
    "            # Forward pass\n",
    "            y_pred = net(images)\n",
    "            y_pred2 = mdl(images)\n",
    "            loss = F.mse_loss(sf[hyper_params[\"stage\"]].features, sf2[hyper_params[\"stage\"]].features)\n",
    "            val.append(loss.item())\n",
    "\n",
    "    val_loss = sum(val) / len(val)\n",
    "    val_loss_list.append(val_loss)\n",
    "    print('epoch : ', epoch + 1, ' / ', hyper_params[\"num_epochs\"], ' | TL : ', train_loss, ' | VL : ', val_loss)\n",
    "\n",
    "    if val_loss < min_val :\n",
    "        print('saving model')\n",
    "        min_val = val_loss\n",
    "        torch.save(net.state_dict(), filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# learn = Learner(data, net, metrics = accuracy)\n",
    "net.cpu()\n",
    "net.load_state_dict(torch.load('../saved_models/stage5/model0.pt', map_location = 'cpu'))\n",
    "net = net.cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.plot(range(hyper_params[\"num_epochs\"]), train_loss_list, label = 'train_loss') \n",
    "plt.plot(range(hyper_params[\"num_epochs\"]), val_loss_list, label = 'val_loss')\n",
    "plt.legend()\n",
    "plt.savefig('../figures/stage5/train_loss.jpg')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Training the classifier only after stage-wise training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "hyper_params = {\n",
    "    \"stage\": 5,\n",
    "    \"repeated\": 3,\n",
    "    \"num_classes\": 10,\n",
    "    \"batch_size\": 64,\n",
    "    \"num_epochs\": 100,\n",
    "    \"learning_rate\": 1e-4\n",
    "}\n",
    "\n",
    "class Flatten(nn.Module) :\n",
    "    def forward(self, input):\n",
    "        return input.view(input.size(0), -1)\n",
    "\n",
    "def conv2(ni, nf) : \n",
    "    return conv_layer(ni, nf, stride = 2)\n",
    "\n",
    "class ResBlock(nn.Module):\n",
    "    def __init__(self, nf):\n",
    "        super().__init__()\n",
    "        self.conv1 = conv_layer(nf,nf)\n",
    "        \n",
    "    def forward(self, x): \n",
    "        return (x + self.conv1(x))\n",
    "\n",
    "def conv_and_res(ni, nf): \n",
    "    return nn.Sequential(conv2(ni, nf), ResBlock(nf))\n",
    "\n",
    "def conv_(nf) : \n",
    "    return nn.Sequential(conv_layer(nf, nf), ResBlock(nf))\n",
    "    \n",
    "net = nn.Sequential(\n",
    "    conv_layer(3, 64, ks = 7, stride = 2, padding = 3),\n",
    "    nn.MaxPool2d(3, 2, padding = 1),\n",
    "    conv_(64),\n",
    "    conv_and_res(64, 128),\n",
    "    conv_and_res(128, 256),\n",
    "    AdaptiveConcatPool2d(),\n",
    "    Flatten(),\n",
    "    nn.Linear(2 * 256, 128),\n",
    "    nn.Linear(128, hyper_params[\"num_classes\"])\n",
    ")\n",
    "\n",
    "net.cpu()\n",
    "net.load_state_dict(torch.load('../saved_models/small_stage4/model1.pt', map_location = 'cpu'))\n",
    "\n",
    "if torch.cuda.is_available() : \n",
    "    net = net.cuda()\n",
    "    print('Model on GPU')\n",
    "    \n",
    "for name, param in net.named_parameters() : \n",
    "    print(name, param.shape)\n",
    "    param.requires_grad = False\n",
    "    if name[0] == '7' or name[0] == '8':\n",
    "        param.requires_grad = True\n",
    "    print(param.requires_grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.982\n",
      "0.934\n"
     ]
    }
   ],
   "source": [
    "def _get_accuracy(dataloader, Net):\n",
    "    total = 0\n",
    "    correct = 0\n",
    "    Net.eval()\n",
    "    for i, (images, labels) in enumerate(dataloader):\n",
    "        images = torch.autograd.Variable(images).float()\n",
    "        labels = torch.autograd.Variable(labels).float()\n",
    "        \n",
    "        if torch.cuda.is_available() : \n",
    "            images = images.cuda()\n",
    "            labels = labels.cuda()\n",
    "\n",
    "        outputs = Net.forward(images)\n",
    "        outputs = F.log_softmax(outputs, dim = 1)\n",
    "\n",
    "        _, pred_ind = torch.max(outputs, 1)\n",
    "        \n",
    "        # converting to numpy arrays\n",
    "        labels = labels.data.cpu().numpy()\n",
    "        pred_ind = pred_ind.data.cpu().numpy()\n",
    "        \n",
    "        # get difference\n",
    "        diff_ind = labels - pred_ind\n",
    "        # correctly classified will be 1 and will get added\n",
    "        # incorrectly classified will be 0\n",
    "        correct += np.count_nonzero(diff_ind == 0)\n",
    "        total += len(diff_ind)\n",
    "\n",
    "    accuracy = correct / total\n",
    "    # print(len(diff_ind))\n",
    "    return accuracy\n",
    "\n",
    "net.cpu()\n",
    "net.load_state_dict(torch.load('../saved_models/large_classifier/model0.pt', map_location = 'cpu'))\n",
    "net.cuda()\n",
    "print(_get_accuracy(data.valid_dl, net))\n",
    "net.cpu()\n",
    "net.load_state_dict(torch.load('../saved_models/large_no_teacher/model0.pt', map_location = 'cpu'))\n",
    "net.cuda()\n",
    "print(_get_accuracy(data.valid_dl, net))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "optimizer = torch.optim.Adam(net.parameters(), lr = hyper_params[\"learning_rate\"])\n",
    "total_step = len(data.train_ds) // hyper_params[\"batch_size\"]\n",
    "train_loss_list = list()\n",
    "val_loss_list = list()\n",
    "min_val = 0\n",
    "for epoch in range(hyper_params[\"num_epochs\"]):\n",
    "    trn = []\n",
    "    net.train()\n",
    "    for i, (images, labels) in enumerate(data.train_dl) :\n",
    "        if torch.cuda.is_available():\n",
    "            images = torch.autograd.Variable(images).cuda().float()\n",
    "            labels = torch.autograd.Variable(labels).cuda()\n",
    "        else : \n",
    "            images = torch.autograd.Variable(images).float()\n",
    "            labels = torch.autograd.Variable(labels)\n",
    "\n",
    "        y_pred = net(images)\n",
    "\n",
    "        loss = F.cross_entropy(y_pred, labels)\n",
    "        trn.append(loss.item())\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "#         torch.nn.utils.clip_grad_value_(net.parameters(), 10)\n",
    "        optimizer.step()\n",
    "\n",
    "        if i % 50 == 49 :\n",
    "            print('epoch = ', epoch, ' step = ', i + 1, ' of total steps ', total_step, ' loss = ', loss.item())\n",
    "\n",
    "    train_loss = (sum(trn) / len(trn))\n",
    "    train_loss_list.append(train_loss)\n",
    "\n",
    "    net.eval()\n",
    "    val = []\n",
    "    with torch.no_grad() :\n",
    "        for i, (images, labels) in enumerate(data.valid_dl) :\n",
    "            if torch.cuda.is_available():\n",
    "                images = torch.autograd.Variable(images).cuda().float()\n",
    "                labels = torch.autograd.Variable(labels).cuda()\n",
    "            else : \n",
    "                images = torch.autograd.Variable(images).float()\n",
    "                labels = torch.autograd.Variable(labels)\n",
    "\n",
    "            # Forward pass\n",
    "            y_pred = net(images)\n",
    "            \n",
    "            loss = F.cross_entropy(y_pred, labels)\n",
    "            val.append(loss.item())\n",
    "\n",
    "    val_loss = sum(val) / len(val)\n",
    "    val_loss_list.append(val_loss)\n",
    "    val_acc = _get_accuracy(data.valid_dl, net)\n",
    "\n",
    "    print('epoch : ', epoch + 1, ' / ', hyper_params[\"num_epochs\"], ' | TL : ', train_loss, ' | VL : ', val_loss, ' | VA : ', val_acc * 100)\n",
    "\n",
    "    if (val_acc * 100) > min_val :\n",
    "        print('saving model')\n",
    "        min_val = val_acc * 100\n",
    "        torch.save(net.state_dict(), '../saved_models/small_classifier/model1.pt')savename"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.888\n",
      "0.8951\n"
     ]
    }
   ],
   "source": [
    "net = resnet14_cifar()\n",
    "net.cpu()\n",
    "net.load_state_dict(torch.load('../saved_models/cifar10/medium_classifier/model0.pt', map_location = 'cpu'))\n",
    "net.cuda()\n",
    "print(_get_accuracy(data.valid_dl, net))\n",
    "\n",
    "net.cpu()\n",
    "net.load_state_dict(torch.load('../saved_models/cifar10/resnet14_classifier/model0.pt', map_location = 'cpu'))\n",
    "net.cuda()\n",
    "print(_get_accuracy(data.valid_dl, net))\n",
    "\n",
    "net = resnet26_cifar()\n",
    "net.cpu()\n",
    "net.load_state_dict(torch.load('../saved_models/cifar10/resnet26_pretraining/model0.pt', map_location = 'cpu'))\n",
    "net.cuda()\n",
    "print(_get_accuracy(data.valid_dl, net))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "from fastai.vision import *\n",
    "import torch\n",
    "from torchsummary import summary\n",
    "torch.cuda.set_device(0)\n",
    "\n",
    "for repeated in range(0, 1) : \n",
    "    torch.manual_seed(repeated)\n",
    "    torch.cuda.manual_seed(repeated)\n",
    "\n",
    "    # stage should be in 0 to 5 (5 for classifier stage)\n",
    "    hyper_params = {\n",
    "        \"stage\": 5,\n",
    "        \"repeated\": repeated,\n",
    "        \"num_classes\": 10,\n",
    "        \"batch_size\": 64,\n",
    "        \"num_epochs\": 100,\n",
    "        \"learning_rate\": 1e-4\n",
    "    }\n",
    "\n",
    "    path = untar_data(URLs.IMAGENETTE)\n",
    "    tfms = get_transforms(do_flip=False)\n",
    "    data = ImageDataBunch.from_folder(path, train = 'train', valid = 'val', bs = hyper_params[\"batch_size\"], size = 224, ds_tfms = tfms).normalize(imagenet_stats)\n",
    "    \n",
    "    class Flatten(nn.Module) :\n",
    "        def forward(self, input):\n",
    "            return input.view(input.size(0), -1)\n",
    "\n",
    "    def conv2(ni, nf) : \n",
    "        return conv_layer(ni, nf, stride = 2)\n",
    "\n",
    "    class ResBlock(nn.Module):\n",
    "        def __init__(self, nf):\n",
    "            super().__init__()\n",
    "            self.conv1 = conv_layer(nf,nf)\n",
    "\n",
    "        def forward(self, x): \n",
    "            return (x + self.conv1(x))\n",
    "\n",
    "    def conv_and_res(ni, nf): \n",
    "        return nn.Sequential(conv2(ni, nf), ResBlock(nf))\n",
    "\n",
    "    def conv_(nf) : \n",
    "        return nn.Sequential(conv_layer(nf, nf), ResBlock(nf))\n",
    "\n",
    "    net = nn.Sequential(\n",
    "        conv_layer(3, 64, ks = 7, stride = 2, padding = 3),\n",
    "        nn.MaxPool2d(3, 2, padding = 1),\n",
    "        conv_(64),\n",
    "        conv_and_res(64, 128),\n",
    "        conv_and_res(128, 256),\n",
    "        AdaptiveConcatPool2d(),\n",
    "        Flatten(),\n",
    "        nn.Linear(2 * 256, 128),\n",
    "        nn.Linear(128, hyper_params[\"num_classes\"])\n",
    "    )\n",
    "\n",
    "    net.cpu()\n",
    "    filename = '../saved_models/small_stage4/model' + str(repeated) + '.pt'\n",
    "    net.load_state_dict(torch.load(filename, map_location = 'cpu'))\n",
    "\n",
    "    if torch.cuda.is_available() : \n",
    "        net = net.cuda()\n",
    "        print('Model on GPU')\n",
    "\n",
    "    for name, param in net.named_parameters() : \n",
    "        param.requires_grad = False\n",
    "        if name[0] == '7' or name[0] == '8':\n",
    "            param.requires_grad = True\n",
    "        \n",
    "    optimizer = torch.optim.Adam(net.parameters(), lr = hyper_params[\"learning_rate\"])\n",
    "    total_step = len(data.train_ds) // hyper_params[\"batch_size\"]\n",
    "    train_loss_list = list()\n",
    "    val_loss_list = list()\n",
    "    min_val = 0\n",
    "    savename = '../saved_models/small_classifier/model' + str(repeated) + '.pt'\n",
    "    for epoch in range(hyper_params[\"num_epochs\"]):\n",
    "        trn = []\n",
    "        net.train()\n",
    "        for i, (images, labels) in enumerate(data.train_dl) :\n",
    "            if torch.cuda.is_available():\n",
    "                images = torch.autograd.Variable(images).cuda().float()\n",
    "                labels = torch.autograd.Variable(labels).cuda()\n",
    "            else : \n",
    "                images = torch.autograd.Variable(images).float()\n",
    "                labels = torch.autograd.Variable(labels)\n",
    "\n",
    "            y_pred = net(images)\n",
    "\n",
    "            loss = F.cross_entropy(y_pred, labels)\n",
    "            trn.append(loss.item())\n",
    "\n",
    "            optimizer.zero_grad()\n",
    "            loss.backward()\n",
    "    #         torch.nn.utils.clip_grad_value_(net.parameters(), 10)\n",
    "            optimizer.step()\n",
    "\n",
    "            if i % 50 == 49 :\n",
    "                print('epoch = ', epoch, ' step = ', i + 1, ' of total steps ', total_step, ' loss = ', loss.item())\n",
    "\n",
    "        train_loss = (sum(trn) / len(trn))\n",
    "        train_loss_list.append(train_loss)\n",
    "\n",
    "        net.eval()\n",
    "        val = []\n",
    "        with torch.no_grad() :\n",
    "            for i, (images, labels) in enumerate(data.valid_dl) :\n",
    "                if torch.cuda.is_available():\n",
    "                    images = torch.autograd.Variable(images).cuda().float()\n",
    "                    labels = torch.autograd.Variable(labels).cuda()\n",
    "                else : \n",
    "                    images = torch.autograd.Variable(images).float()\n",
    "                    labels = torch.autograd.Variable(labels)\n",
    "\n",
    "                # Forward pass\n",
    "                y_pred = net(images)\n",
    "\n",
    "                loss = F.cross_entropy(y_pred, labels)\n",
    "                val.append(loss.item())\n",
    "\n",
    "        val_loss = sum(val) / len(val)\n",
    "        val_loss_list.append(val_loss)\n",
    "        val_acc = _get_accuracy(data.valid_dl, net)\n",
    "\n",
    "        print('epoch : ', epoch + 1, ' / ', hyper_params[\"num_epochs\"], ' | TL : ', train_loss, ' | VL : ', val_loss, ' | VA : ', val_acc * 100)\n",
    "\n",
    "        if (val_acc * 100) > min_val :\n",
    "            print('saving model')\n",
    "            min_val = val_acc * 100\n",
    "            torch.save(net.state_dict(), savename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (ak_fastai)",
   "language": "python",
   "name": "ak_fastai"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
